package com.example.sxt_practise.net;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;

/**
 * @author gaofeng
 * @date 2022年06月09日 21:56
 */
public class SpiderTest {
    public static void main(String[] args) throws IOException {
        // 1.获取URL
        URL jdUrl = new URL("https://www.baidu.com");
//        InputStream inputStream = jdUrl.openStream() ;

        // 模拟浏览器请求, 有些网站不能用jdUrl.openStream()爬,用此方式
        HttpURLConnection connection = (HttpURLConnection) jdUrl.openConnection();
        connection.setRequestMethod("GET");
        connection.setRequestProperty("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36");

        try (InputStream inputStream =  connection.getInputStream();OutputStream outputStream = new BufferedOutputStream(new FileOutputStream("baidu.txt"));){
            // 2.下载资源
            byte[] flush = new byte[1024 * 2] ;
            int len = -1 ;
            while ((len = inputStream.read(flush)) != -1) {
                outputStream.write(flush, 0 , len);
            }
            // 3.分析
        } catch (IOException e) {
            extracted(e);
        }
    }

    private static void extracted(Exception e) {
        e.printStackTrace();
    }
}
